# -*- coding: utf-8 -*-

import random
from scrapy.downloadermiddlewares.retry import RetryMiddleware, response_status_message
import logging
from scrapy_pyppeteer import PyppeteerRequest
from copy import copy
import json

logger = logging.getLogger(__name__)


class UserAgentMiddleware(object):
    def process_request(self, request, spider):
        user_agent = random.choice(spider.settings['USER_AGENTS'])
        request.headers['User-Agent'] = user_agent


class CustomRetryMiddleware(RetryMiddleware):
    def process_response(self, request, response, spider):
        if request.meta.get('dont_retry', False):
            return response
        if response.status in self.retry_http_codes:
            reason = response_status_message(response.status)
            return self._retry(request, reason, spider) or response
        if spider.name == 'anjuke':
            if response.status in [301, 302, 307]:
                if 'px1-pv2' in request.url:
                    retries = request.meta.get('retry_times', 0) + 1
                    logger.warning(
                        '{}:{} 被重定向到验证码页面，更换代理ip尝试重试，第{}次重试'.format(spider.name, request.url, retries))
                    reason = response_status_message(response.status)
                    pyp_request = PyppeteerRequest(request.url, callback=request.callback, pretend=True,
                                                   meta=request.meta,
                                                   wait_for='p.item-descript',
                                                   ignore_resource_types=['image', 'media'], )
                    return self._retry(pyp_request, reason, spider) or response
        if spider.name == 'landchina':
            if request.method == 'POST':
                # 供地结果列表
                if request.url == 'https://api.landchina.com/tGdxm/result/list':
                    req_dict = json.loads((request.body.decode()))
                    page_num = req_dict['pageNum']
                    try:
                        resp_dict = json.loads(response.body.decode())
                        data_dict = resp_dict.get('data', None)
                        current_page = data_dict['pageNum']
                        total_page = data_dict['pages']
                        next_page = data_dict['nextPage']
                        assert ((page_num == current_page and next_page == page_num + 1) or (page_num == total_page))
                    except:
                        retries = request.meta.get('retry_times', 0) + 1
                        logger.error(f'供地结果-第{page_num}页-页面被反爬，尝试重试，第{retries}次重试')
                        reason = response_status_message(response.status)
                        return self._retry(request, reason, spider) or response
                # 供地结果详情
                if request.url == 'https://api.landchina.com/tGdxm/result/detail':
                    req_dict = json.loads((request.body.decode()))
                    gdGuid = req_dict['gdGuid']
                    try:
                        resp_dict = json.loads(response.body.decode())
                        data_dict = resp_dict.get('data', None)
                        assert gdGuid == data_dict['gdGuid']
                    except:
                        retries = request.meta.get('retry_times', 0) + 1
                        logger.error(f'供地结果-{gdGuid}-页面被反爬，第{retries}次重试')
                        reason = response_status_message(response.status)
                        return self._retry(request, reason, spider) or response
                # 出让公告列表
                if request.url == 'https://api.landchina.com/tGygg/transfer/list':
                    req_dict = json.loads((request.body.decode()))
                    page_num = req_dict['pageNum']
                    try:
                        resp_dict = json.loads(response.body.decode())
                        data_dict = resp_dict.get('data', None)
                        current_page = data_dict['pageNum']
                        total_page = data_dict['pages']
                        next_page = data_dict['nextPage']
                        assert ((page_num == current_page and next_page == page_num + 1) or (page_num == total_page))
                    except:
                        retries = request.meta.get('retry_times', 0) + 1
                        logger.error(f'出让公告-第{page_num}页-页面被反爬，尝试重试，第{retries}次重试')
                        reason = response_status_message(response.status)
                        return self._retry(request, reason, spider) or response
                # 出让公告详情
                if request.url == 'https://api.landchina.com/tGygg/transfer/detail':
                    req_dict = json.loads((request.body.decode()))
                    gyggGuid = req_dict['gyggGuid']
                    try:
                        resp_dict = json.loads(response.body.decode())
                        relate_dict = resp_dict.get('relate', list())[0]
                        assert gyggGuid == relate_dict['gyggGuid']
                    except:
                        retries = request.meta.get('retry_times', 0) + 1
                        logger.error(f'出让公告-{gyggGuid}-页面被反爬，第{retries}次重试')
                        reason = response_status_message(response.status)
                        return self._retry(request, reason, spider) or response
        return response

    def process_exception(self, request, exception, spider):
        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
                and not request.meta.get('dont_retry', False):
            return self._retry(request, exception, spider)


class OuterNetProxyMiddleware(object):
    def process_request(self, request, spider):
        key_name, proxy_ip = spider.redis_conn.brpop('proxy', timeout=30 * 60)
        proxy_dict = {
            "http": "http://hifo:hifo1996@{}:65500".format(proxy_ip),
            "https": "https://hifo:hifo1996@{}:65500".format(proxy_ip)
        }
        request.meta['proxy'] = proxy_dict['http'] if request.url.startswith("http://") else proxy_dict['https']
